# -*- coding: utf-8 -*-
import scrapy
import json
from scrapy.http.response.html import HtmlResponse
from scrapy.http import Request
from scrapy_demo01.items import YouKuFilm2
# import logging
# logging.basicConfig(format='%(asctime)s %(threadName)s-%(thread)d %(levelname)s: %(message)s', level=logging.INFO)


class YoukuSpider2Spider(scrapy.Spider):
    name = 'youku_spider2'
    allowed_domains = ['youku.com']
    # 爬取3页
    # start_urls = ['https://list.youku.com/category/page?c=96&g=%E7%A7%91%E5%B9%BB&type=show&p=' + str(i) for i in range(1, 4)]
    # 使用parse返回Request爬取3页
    start_urls = ['https://list.youku.com/category/page?c=96&g=%E7%A7%91%E5%B9%BB&type=show&p=']
    page = 1

    # parse方法可以返回item对象，也可以返回Request对象，继续爬取
    def parse(self, response: HtmlResponse):
        # scrapy crawl -L INFO youku_spider2
        # self.logger.info("GET : %s", response.url)
        print("GET :", response.url)
        self.page += 1
        if response.status == 200:
            data = json.loads(response.text)
            if data['success']:
                for f in data['data']:
                    yield YouKuFilm2(title=f.get('title'), subTitle=f.get('subTitle')
                                     , summary=f.get('summary'))
                if self.page <= 3:  # 使用Request继续获取两页， 传递callback函数处理请求，函数可以不是self.parse，只要是第一个参数接收Response就可以
                    yield Request(self.start_urls[0] + str(self.page), self.parse)
        else:
            print('response error ->', response)
